/*
 *  EvaluationBasedOnDocs.java
 *
 *  Yaoyong Li 22/03/2007
 *
 *  $Id: EvaluationBasedOnDocs.java, v 1.0 2007-03-22 12:58:16 +0000 yaoyong $
 */
package gate.learning;

import gate.Annotation;
import gate.AnnotationSet;
import gate.Corpus;
import gate.Document;
import gate.Factory;
import gate.FeatureMap;
import gate.creole.ResourceInstantiationException;
import gate.util.AnnotationDiffer;
import gate.util.BomStrippingInputStreamReader;
import gate.util.GateException;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.Set;

/**
 * Do evaluation by splitting the documents into training and testing datasets.
 * Two methods of splitting are implemented, namely k-fold and hold-out test.
 */
public class EvaluationBasedOnDocs {

    /** Corpus referring to the corpus used as data. */
    Corpus corpusOn;
    /** Number of documents in the corpus. */
    int numDoc;
    /**
     * Showing if one document used for training or testing in one evaluation.
     */
    boolean[] isUsedForTraining;
    /** The sub-directory for storing the data file produced by ML api. */
    File wdResults;
    /** Name of the annotation set as input. */
    String inputASName;
    /** Storing the macro averaged overall F-measure results of the evaluation. */
    public EvaluationMeasuresComputation macroMeasuresOfResults = new EvaluationMeasuresComputation();
    /** Storing the macro averaged results of every label. */
    HashMap labels2MMR = new HashMap();
    /**
     * Label to number of the runs having that label, used for macro-averaged for
     * each label.
     */
    HashMap labels2RunsNum = new HashMap();
    /**
     * Label to number of the instances with that label, averaged over the number
     * of runs for that label.
     */
    HashMap labels2InstNum = new HashMap();

    /** Constructor. */
    public EvaluationBasedOnDocs(Corpus corpus, File wdRes, String inputAsN) {
        corpusOn = corpus;
        numDoc = corpus.size();
        isUsedForTraining = new boolean[numDoc];
        wdResults = wdRes;
        inputASName = inputAsN;
    }

    /**
     * Main method for evaluation.
     *
     * @throws IOException
     */
    public void evaluation(LearningEngineSettings learningSettings,
            LightWeightLearningApi lightWeightApi) throws GateException, IOException {
        // k-fold
        if (learningSettings.evaluationconfig.mode == EvaluationConfiguration.kfold) {
            kfoldEval(learningSettings, lightWeightApi);
        }
        // Hold-out testing
        else if (learningSettings.evaluationconfig.mode == EvaluationConfiguration.split) {
            holdoutEval(learningSettings, lightWeightApi);
        } else {
            throw new GateException("The evaluation configuration mode as "
                    + learningSettings.evaluationconfig.mode + " is not implemented!");
        }
        StringBuffer logMes = new StringBuffer();
        logMes.append("\n*** Averaged results for each label over "
                + learningSettings.evaluationconfig.kk + " runs as:\n");
        logMes.append(printFmeasureForEachLabel(labels2InstNum, labels2MMR));
        logMes.append("\nOverall results as:\n");
        logMes.append(macroMeasuresOfResults.printResults());
        LogService.logMessage(logMes.toString(), 1);
        if (LogService.maxVerbosityLevel > 0) {
            System.out.println(logMes);
        }
    }

    /** K-fold evaluation. */
    public void kfoldEval(LearningEngineSettings learningSettings,
            LightWeightLearningApi lightWeightApi) throws GateException {
        int k = learningSettings.evaluationconfig.kk;
        LogService.logMessage("K-fold evaluation: k=" + k, 1);
        int lenPerFold = (new Double(Math.floor((double) numDoc / k))).intValue();
        if (lenPerFold < 1) {
            lenPerFold = 1;
        }
        int beginIndex, endIndex;
        if (LogService.maxVerbosityLevel > 0) {
            System.out.println("Kfold k=" + new Integer(k) + ", numDoc="
                    + new Integer(numDoc) + ", len=" + new Integer(lenPerFold) + ".");
        }
        LogService.logMessage("Kfold k=" + new Integer(k) + ", numDoc="
                + new Integer(numDoc) + ", len=" + new Integer(lenPerFold) + ".", 1);
        for (int nr = 0; nr < k; ++nr) {
            EvaluationMeasuresComputation measuresOfResults = new EvaluationMeasuresComputation();
            // Label to measure of result of the label
            HashMap labels2MR = new HashMap();
            beginIndex = nr * lenPerFold;
            endIndex = (nr + 1) * lenPerFold;
            if (endIndex > numDoc) {
                endIndex = numDoc;
            }
            if (beginIndex > endIndex) {
                beginIndex = endIndex;
            }
            int i;
            for (i = 0; i < beginIndex; ++i) {
                isUsedForTraining[i] = true;
            }
            for (i = beginIndex; i < endIndex; ++i) {
                isUsedForTraining[i] = false;
            }
            for (i = endIndex; i < numDoc; ++i) {
                isUsedForTraining[i] = true;
            }
            StringBuffer logMes = new StringBuffer();
            int nr0 = nr + 1;
            logMes.append("\n*** Fold " + nr0 + "\n");
            logMes.append("Number of docs for training: "
                    + (int) (numDoc - endIndex + beginIndex) + "\n");
            int ik = 0;
            for (i = 0; i < numDoc; ++i) {
                if (isUsedForTraining[i]) {
                    ++ik;
                    Document toProcess = (Document) corpusOn.get(i);
                    logMes.append(ik + " " + toProcess.getName() + "\n");
                    if (toProcess.getDataStore() != null
                            && corpusOn.getDataStore() != null) {
                        Factory.deleteResource(toProcess);
                    }
                }
            }
            logMes.append("Number of docs for application: "
                    + (int) (endIndex - beginIndex) + "\n");
            ik = 0;
            for (i = 0; i < numDoc; ++i) {
                if (!isUsedForTraining[i]) {
                    ++ik;
                    Document toProcess = (Document) corpusOn.get(i);
                    logMes.append(ik + " " + toProcess.getName() + "\n");
                    if (toProcess.getDataStore() != null
                            && corpusOn.getDataStore() != null) {
                        Factory.deleteResource(toProcess);
                    }
                }
            }
            LogService.logMessage(logMes.toString(), 1);
            if (LogService.maxVerbosityLevel > 1) {
                System.out.println(logMes);
            }
            // call the training or application
            oneRun(learningSettings, lightWeightApi, isUsedForTraining,
                    measuresOfResults, labels2MR, labels2InstNum);
            // Add to the macro averaged figures
            add2MacroMeasure(measuresOfResults, labels2MR, macroMeasuresOfResults,
                    labels2MMR, labels2RunsNum);
        }
        macroMeasuresOfResults.macroAverage(k);
        for (Object obj : labels2MMR.keySet()) {
            int num = new Integer(labels2RunsNum.get(obj).toString()).intValue();
            ((EvaluationMeasuresComputation) labels2MMR.get(obj)).macroAverage(num);
            num = (int) (new Float(labels2InstNum.get(obj).toString()).floatValue() / num);
            labels2InstNum.put(obj, new Integer(num));
        }
    }

    /** Hold-out testing. */
    public void holdoutEval(LearningEngineSettings learningSettings,
            LightWeightLearningApi lightWeightApi) throws GateException {
        int k = learningSettings.evaluationconfig.kk;
        LogService.logMessage("Hold-out test: runs=" + k
                + ", ratio of training docs is "
                + learningSettings.evaluationconfig.ratio, 1);
        int trainingNum = (new Double(Math.floor((numDoc * learningSettings.evaluationconfig.ratio))).intValue());
        if (trainingNum > numDoc) {
            trainingNum = numDoc;
        }
        LogService.logMessage("Split, k=" + new Integer(k) + ", trainingNum="
                + new Integer(trainingNum) + ".", 1);
        if (LogService.maxVerbosityLevel > 0) {
            System.out.println("Hold-out test: runs=" + k
                    + ", ratio of training docs is "
                    + learningSettings.evaluationconfig.ratio);
            System.out.println("Split, k=" + new Integer(k) + ", trainingNum="
                    + new Integer(trainingNum) + ".");
        }
        int testNum = numDoc - trainingNum;
        Random randGenerator = new Random(1000);
        for (int nr = 0; nr < k; ++nr) {
            EvaluationMeasuresComputation measuresOfResults = new EvaluationMeasuresComputation();
            // Label to measure of result of the label
            HashMap labels2MR = new HashMap();
            // Select the training examples randomly from the data
            int[] indexRand = new int[testNum];
            for (int i = 0; i < testNum; ++i) {
                int newNum = 0;
                boolean isDuplicate = true;
                int maxNumSel = 0;
                while (isDuplicate && maxNumSel < 1000) {
                    newNum = randGenerator.nextInt(numDoc);
                    isDuplicate = false;
                    for (int j = 0; j < i; ++j) {
                        if (indexRand[j] == newNum) {
                            isDuplicate = true;
                            break;
                        }
                    }
                }
                if (isDuplicate) {
                    // If cannot select a non-duplicate index in
                    // 1000 times, selected the one
                    boolean isOk = false;
                    for (int j = 0; j < numDoc; ++j) {
                        for (int j1 = 0; j1 < i; ++j1) {
                            if (indexRand[j1] != j) {
                                newNum = j;
                                isOk = true;
                                break;
                            }
                        }
                        if (isOk) {
                            break;
                        }
                    }
                }
                if (LogService.maxVerbosityLevel > 1) {
                    System.out.println("i=" + new Integer(i) + ", newNum="
                            + new Integer(newNum));
                }
                indexRand[i] = newNum;
            }
            for (int i = 0; i < numDoc; ++i) {
                isUsedForTraining[i] = true;
            }
            for (int i = 0; i < testNum; ++i) {
                isUsedForTraining[indexRand[i]] = false;
            }
            StringBuffer logMes = new StringBuffer();
            int nr0 = nr + 1;
            logMes.append("\n*** Run " + nr0 + "\n");
            logMes.append("Number of docs for training: " + (int) (numDoc - testNum)
                    + "\n");
            int ik = 0;
            for (int i = 0; i < numDoc; ++i) {
                if (isUsedForTraining[i]) {
                    ++ik;
                    Document toProcess = (Document) corpusOn.get(i);
                    logMes.append(ik + " " + toProcess.getName() + "\n");
                    if (toProcess.getDataStore() != null
                            && corpusOn.getDataStore() != null) {
                        Factory.deleteResource(toProcess);
                    }
                }
            }
            logMes.append("Number of docs for application: " + testNum + "\n");
            ik = 0;
            for (int i = 0; i < numDoc; ++i) {
                if (!isUsedForTraining[i]) {
                    ++ik;
                    Document toProcess = (Document) corpusOn.get(i);
                    logMes.append(ik + " " + toProcess.getName() + "\n");
                    if (toProcess.getDataStore() != null
                            && corpusOn.getDataStore() != null) {
                        Factory.deleteResource(toProcess);
                    }
                }
            }
            LogService.logMessage(logMes.toString(), 1);
            if (LogService.maxVerbosityLevel > 1) {
                System.out.println(logMes);
            }
            // One run, call the training and application and do evaluation
            oneRun(learningSettings, lightWeightApi, isUsedForTraining,
                    measuresOfResults, labels2MR, labels2InstNum);
            // Add to the macro averaged figures
            add2MacroMeasure(measuresOfResults, labels2MR, macroMeasuresOfResults,
                    labels2MMR, labels2RunsNum);
        }
        macroMeasuresOfResults.macroAverage(k);
        for (Object obj : labels2MMR.keySet()) {
            int num = new Integer(labels2RunsNum.get(obj).toString()).intValue();
            ((EvaluationMeasuresComputation) labels2MMR.get(obj)).macroAverage(num);
            // Averaged the number of instances for all the runs
            num = (int) (new Float(labels2InstNum.get(obj).toString()).floatValue() / num);
            labels2InstNum.put(obj, new Integer(num));
        }
    }

    /**
     * One run of the evaluation: training, testing and measuring results.
     *
     * @throws
     * @throws UnsupportedEncodingException
     */
    private void oneRun(LearningEngineSettings learningSettings,
            LightWeightLearningApi lightWeightApi, boolean isUsedForTraining[],
            EvaluationMeasuresComputation measuresOfResults, HashMap labels2MR,
            HashMap labels2InstNum) throws GateException {
        // first learning using the document for training
        // empty the data file
        emptyDatafile(wdResults, true);
        lightWeightApi.labelsAndId.clearAllData();
        lightWeightApi.featuresList.clearAllData();
        boolean isTraining = true;
        int numDoc = 0;
        BufferedWriter outNLPFeatures = null;
        BufferedReader inNLPFeatures = null;
        BufferedWriter outFeatureVectors = null;
        try {
            outNLPFeatures = new BufferedWriter(new OutputStreamWriter(
                    new FileOutputStream(new File(wdResults,
                    ConstantParameters.FILENAME_FEATURES_CSV)), "UTF-8"));
            for (int i = 0; i < corpusOn.size(); ++i) {
                if (isUsedForTraining[i]) {
                    Document toProcess = (Document) corpusOn.get(i);
                    lightWeightApi.annotations2NLPFeatures(toProcess, numDoc,
                            outNLPFeatures, isTraining, learningSettings);
                    if (toProcess.getDataStore() != null
                            && corpusOn.getDataStore() != null) {
                        Factory.deleteResource(toProcess);
                    }
                    ++numDoc;
                }
            }
            outNLPFeatures.flush();
            outNLPFeatures.close();
            lightWeightApi.finishFVs(wdResults, numDoc, isTraining, learningSettings);
            // Open the normal NLP feature file.
            inNLPFeatures = new BomStrippingInputStreamReader(
                    new FileInputStream(new File(wdResults,
                    ConstantParameters.FILENAME_FEATURES_CSV)), "UTF-8");
            outFeatureVectors = new BufferedWriter(new OutputStreamWriter(
                    new FileOutputStream(new File(wdResults,
                    ConstantParameters.FILENAMEOFFeatureVectorData)), "UTF-8"));
            lightWeightApi.nlpfeatures2FVs(wdResults, inNLPFeatures,
                    outFeatureVectors, numDoc, isTraining, learningSettings);
            inNLPFeatures.close();
            outFeatureVectors.flush();
            outFeatureVectors.close();
            // outFeatureVectors.close();
            // read the fv data from total file and write them into fv files for
            // training and application, respectively
      /*
             * BufferedReader inFVs = new BufferedReader(new InputStreamReader(new
             * FileInputStream(new File(wdResults,
             * ConstantParameters.FILENAMEOFFeatureVectorTotalData)), "UTF-8"));
             * BufferedWriter outFVs = new BufferedWriter(new OutputStreamWriter(new
             * FileOutputStream( new
             * File(wdResults,ConstantParameters.FILENAMEOFFeatureVectorData)),
             * "UTF-8")); BufferedWriter outFVsApp = new BufferedWriter(new
             * OutputStreamWriter(new FileOutputStream( new
             * File(wdResults,ConstantParameters.FILENAMEOFFeatureVectorDataApp)),
             * "UTF-8")); for(int i = 0; i < corpusOn.size(); ++i) { String str =
             * inFVs.readLine(); int numLines =
             * Integer.parseInt(str.substring(str.indexOf(ConstantParameters.ITEMSEPARATOR)+1,
             * str.lastIndexOf(ConstantParameters.ITEMSEPARATOR)));
             * if(isUsedForTraining[i]) { outFVs.append(str); outFVs.newLine();
             * for(int j=0; j<numLines; ++j) { outFVs.append(inFVs.readLine());
             * outFVs.newLine(); } ++numDoc; } else { outFVsApp.append(str);
             * outFVsApp.newLine(); for(int j=0; j<numLines; ++j) {
             * outFVsApp.append(inFVs.readLine()); outFVsApp.newLine(); } } }
             * inFVs.close(); outFVs.flush(); outFVs.close(); outFVsApp.flush();
             * outFVsApp.close();
             */
            // if fitering the training data
            if (learningSettings.fiteringTrainingData
                    && learningSettings.filteringRatio > 0.0) {
                lightWeightApi.FilteringNegativeInstsInJava(numDoc, learningSettings);
            }
            // lightWeightApi.trainDirect();
            lightWeightApi.trainingJava(numDoc, learningSettings);
            // then application to the test set
            // First we empty the NLP feature file and feature vector file; but not
            // the list files.
            emptyDatafile(wdResults, false);
            // We have to use two class types for the evaluation purpose
            String classTypeOriginal = null;
            String classTypeTest = null;
            String classFeature = null;
            classTypeOriginal = learningSettings.datasetDefinition.getClassAttribute().getType();
            classTypeTest = classTypeOriginal.concat("Test");
            classFeature = learningSettings.datasetDefinition.getClassAttribute().getFeature();
            learningSettings.datasetDefinition.getClassAttribute().setType(
                    classTypeTest);
            if (LogService.maxVerbosityLevel > 1) {
                System.out.println("classType=" + classTypeOriginal + ", testType="
                        + classTypeTest + ".");
            }
            isTraining = false;
            // numDoc = isUsedForTraining.length - numDoc;
            numDoc = 0;
            outNLPFeatures = new BufferedWriter(new OutputStreamWriter(
                    new FileOutputStream(new File(wdResults,
                    ConstantParameters.FILENAME_FEATURES_CSV)), "UTF-8"));
            for (int i = 0; i < corpusOn.size(); ++i) {
                if (!isUsedForTraining[i]) {
                    Document toProcess = (Document) corpusOn.get(i);
                    lightWeightApi.annotations2NLPFeatures(toProcess,
                            numDoc, outNLPFeatures, isTraining, learningSettings);
                    ++numDoc;
                    if (toProcess.getDataStore() != null
                            && corpusOn.getDataStore() != null) {
                        corpusOn.getDataStore().sync(corpusOn);
                        Factory.deleteResource(toProcess);
                    }
                }
            }
            outNLPFeatures.flush();
            outNLPFeatures.close();
            lightWeightApi.finishFVs(wdResults, numDoc, isTraining, learningSettings);
            inNLPFeatures = new BomStrippingInputStreamReader(
                    new FileInputStream(new File(wdResults,
                    ConstantParameters.FILENAME_FEATURES_CSV)), "UTF-8");
            outFeatureVectors = new BufferedWriter(new OutputStreamWriter(
                    new FileOutputStream(new File(wdResults,
                    ConstantParameters.FILENAMEOFFeatureVectorDataApp)), "UTF-8"));
            lightWeightApi.nlpfeatures2FVs(wdResults, inNLPFeatures,
                    outFeatureVectors, numDoc, isTraining, learningSettings);
            inNLPFeatures.close();
            outFeatureVectors.flush();
            outFeatureVectors.close();
            // lightWeightApi.finishDocAnnotation();
            Corpus corpusTest;
            corpusTest = Factory.newCorpus("testCorpus");
            if (corpusOn.getDataStore() != null) {
                Corpus corpusTestTemp = (Corpus) corpusOn.getDataStore().adopt(corpusTest, null);
                Factory.deleteResource(corpusTest);
                corpusTest = corpusTestTemp;
                corpusTest.getDataStore().sync(corpusTest);
            }
            numDoc = 0;
            for (int i = 0; i < corpusOn.size(); ++i) {
                if (!isUsedForTraining[i]) {
                    Document d = (Document) corpusOn.get(i);
                    corpusTest.add(d);
                    if (corpusTest.getDataStore() != null) {
                        corpusTest.getDataStore().sync(corpusTest);
                        Factory.deleteResource(d);
                    }
                    ++numDoc;
                }
            }
            lightWeightApi.applyModelInJava(corpusTest, 0, corpusTest.size(),
                    classTypeTest, learningSettings, wdResults);
            //unload the documents in the test corpus from memory
      /*for(int i=0; i<corpusTest.size(); ++i) {
            Document toProcess = (Document)corpusOn.get(i);
            if(corpusOn.getDataStore() != null) {
            corpusOn.getDataStore().sync(corpusOn);
            Factory.deleteResource(toProcess);
            }
            }*/
            corpusTest.clear();
            Factory.deleteResource(corpusTest);
            if (corpusTest.getDataStore() != null) {
                corpusTest.getDataStore().delete(corpusTest.getClass().getName(), corpusTest.getLRPersistenceId());
            }

            // Do the evaluation on test using the AnnotationDiff
            // First get all the labels in the training data,
            // so that we can do evaluation on each single label
            // as well as on all labels
            HashMap uniqueLabels = new HashMap();
            for (int i = 0; i < corpusOn.size(); ++i) {
                if (isUsedForTraining[i]) {
                    Document toProcess = (Document) corpusOn.get(i);
                    AnnotationSet keyAnns = getInputAS(toProcess).get(classTypeOriginal);
                    for (Object obj : keyAnns) {
                        if (((Annotation) obj).getFeatures().get(classFeature) != null) {
                            String label = ((Annotation) obj).getFeatures().get(classFeature).toString();
                            if (uniqueLabels.containsKey(label)) {
                                uniqueLabels.put(label, new Integer(new Integer(uniqueLabels.get(label).toString()).intValue() + 1));
                            } else {
                                uniqueLabels.put(label, "1");
                            }
                        }
                    }
                    if (toProcess.getDataStore() != null
                            && corpusOn.getDataStore() != null) {
                        corpusOn.getDataStore().sync(corpusOn);
                        Factory.deleteResource(toProcess);
                    }
                }
            }
            // Then create one evaluationMeasure object for each label
            for (Object obj : uniqueLabels.keySet()) {
                EvaluationMeasuresComputation emc = new EvaluationMeasuresComputation();
                labels2MR.put(obj, emc);
            }
            // Copy the number of instances for each label into the macro one
            for (Object obj : uniqueLabels.keySet()) {
                if (labels2InstNum.containsKey(obj)) {
                    int num = new Integer(uniqueLabels.get(obj).toString()).intValue();
                    labels2InstNum.put(obj, new Integer(new Integer(labels2InstNum.get(
                            obj).toString()).intValue()
                            + num));
                } else {
                    labels2InstNum.put(obj, uniqueLabels.get(obj));
                }
            }
            // Do the evaluation on the test set
            if (learningSettings.datasetDefinition.dataType == DataSetDefinition.RelationData) {
                // For relation type, we cannot use the evaluation method AnnDiff
                // of Gate
                AttributeRelation relAttr = (AttributeRelation) learningSettings.datasetDefinition.getClassAttribute();
                String arg1F = relAttr.getArg1();
                String arg2F = relAttr.getArg2();
                for (int i = 0; i < corpusOn.size(); ++i) {
                    if (!isUsedForTraining[i]) {
                        Document toProcess = (Document) corpusOn.get(i);
                        evaluateAnnotationsRel(toProcess,
                                classTypeOriginal, classTypeTest, classFeature, arg1F, arg2F,
                                uniqueLabels.keySet(), labels2MR);
                        if (toProcess.getDataStore() != null
                                && corpusOn.getDataStore() != null) {
                            corpusOn.getDataStore().sync(corpusOn);
                            Factory.deleteResource(toProcess);
                        }
                    }
                }
            } else {
                /*
                 * for(int i=0; i<corpusOn.size(); ++i) { if(! isUsedForTraining[i]) {
                 * evaluateAnnotations((Document)corpusOn.get(i), classTypeOriginal,
                 * classTypeTest, classFeature, uniqueLabels.keySet(), labels2MR); } }
                 */
                // evaluation on each label using the AnnDiff method of Gate
                for (int i = 0; i < corpusOn.size(); ++i) {
                    if (!isUsedForTraining[i]) {
                        Document toProcess = (Document) corpusOn.get(i);
                        evaluateAnnDiff(toProcess, classTypeOriginal,
                                classTypeTest, classFeature, uniqueLabels.keySet(), labels2MR);
                        if (toProcess.getDataStore() != null
                                && corpusOn.getDataStore() != null) {
                            corpusOn.getDataStore().sync(corpusOn);
                            Factory.deleteResource(toProcess);
                        }
                    }
                }
            }
            // Pool the results of all labels together
            for (Object obj : uniqueLabels.keySet()) {
                measuresOfResults.add((EvaluationMeasuresComputation) labels2MR.get(obj));
            }
            measuresOfResults.computeFmeasure();
            measuresOfResults.computeFmeasureLenient();
            for (Object obj : uniqueLabels.keySet()) {
                EvaluationMeasuresComputation emc = (EvaluationMeasuresComputation) labels2MR.get(obj);
                emc.computeFmeasure();
                emc.computeFmeasureLenient();
            }
            StringBuffer logMes = new StringBuffer();
            logMes.append("Results of this run:\n\n");
            logMes.append(printFmeasureForEachLabel(uniqueLabels, labels2MR));
            logMes.append("\nOverall results(micro-averaged over all labels):\n");
            logMes.append(measuresOfResults.printResults());
            if (LogService.maxVerbosityLevel > 1) {
                System.out.println(logMes);
            }
            LogService.logMessage(logMes.toString(), 1);
            // finally, change the class type back for training,
            // and remove the test annotations
            learningSettings.datasetDefinition.getClassAttribute().setType(
                    classTypeOriginal);
            for (int i = 0; i < corpusOn.size(); ++i) {
                if (!isUsedForTraining[i]) {
                    Document toProcess = (Document) corpusOn.get(i);
                    AnnotationSet annsInput = getInputAS(toProcess);
                    AnnotationSet anns = annsInput.get(classTypeTest);
                    Iterator iter = anns.iterator();
                    while (iter.hasNext()) {
                        annsInput.remove((Annotation) iter.next());
                    }
                    if (toProcess.getDataStore() != null
                            && corpusOn.getDataStore() != null) {
                        corpusOn.getDataStore().sync(corpusOn);
                        Factory.deleteResource(toProcess);
                    }
                }
            }
        } catch (ResourceInstantiationException e) {
            e.printStackTrace();
        } catch (UnsupportedEncodingException e1) {
            e1.printStackTrace();
        } catch (FileNotFoundException e1) {
            e1.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /** Empty the label list and feature list. */
    public static void emptyDatafile(File wdResults, boolean isRemoveList) {
        if (isRemoveList) {
            (new File(wdResults, ConstantParameters.FILENAMEOFNLPFeatureList)).delete();
            (new File(wdResults, ConstantParameters.FILENAMEOFLabelList)).delete();
            (new File(wdResults, ConstantParameters.FILENAMEOFChunkLenStats)).delete();
            (new File(wdResults, ConstantParameters.FILENAMEOFLabelsInData)).delete();
            try {
                (new File(wdResults, ConstantParameters.FILENAMEOFNLPFeatureList)).createNewFile();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        (new File(wdResults, ConstantParameters.FILENAME_FEATURES_CSV)).delete();
        (new File(wdResults, ConstantParameters.FILENAMEOFFeatureVectorData)).delete();
        (new File(wdResults, ConstantParameters.FILENAMEOFNLPDataLabel)).delete();
        System.gc(); // to make effort to delete the files.
    }

    /** Add the F-measure of each label to the overal F-measure */
    public void add2MacroMeasure(EvaluationMeasuresComputation measuresOfResults,
            HashMap labels2MR, EvaluationMeasuresComputation macroMeasuresOfResults,
            HashMap labels2MMR, HashMap labels2RunsNum) {
        macroMeasuresOfResults.add(measuresOfResults);
        // for each label
        for (Object obj : labels2MR.keySet()) {
            String label = obj.toString();
            if (labels2MMR.containsKey(label)) {// the label is in the macro
                // Array already
                ((EvaluationMeasuresComputation) labels2MMR.get(label)).add(((EvaluationMeasuresComputation) (labels2MR.get(label))));
                labels2RunsNum.put(label, new Integer((new Integer(labels2RunsNum.get(
                        label).toString()).intValue() + 1)));
            } else {
                // labels2MMR.put(label, new EvaluationMeasuresComputation());
                EvaluationMeasuresComputation emc = new EvaluationMeasuresComputation();
                emc.add((EvaluationMeasuresComputation) (labels2MR.get(label)));
                labels2MMR.put(label, emc);
                labels2RunsNum.put(label, "1");
            }
        }
    }

    /** Print the F-measure results for each label. */
    public String printFmeasureForEachLabel(HashMap uniqueLabels,
            HashMap labels2MR) {
        StringBuffer logMes = new StringBuffer();
        logMes.append("\nResults of single label:\n");
        // LogService.logMessage("\nResults of single label", 1);
        List labels = new ArrayList(uniqueLabels.keySet());
        Collections.sort(labels);
        for (int i = 0; i < labels.size(); ++i) {
            String labelName = labels.get(i).toString();
            logMes.append(i + " LabelName=" + labelName + ", number of instances="
                    + new Integer(uniqueLabels.get(labelName).toString()) + "\n");
            logMes.append(((EvaluationMeasuresComputation) labels2MR.get(labelName)).printResults());
            // ((EvaluationMeasuresComputation)labels2MR.get(labelName))
            // .printResults(logFileIn);
        }
        return logMes.toString();
    }

    /** Evaluate the test document by using the AnnotationDiff class. */
    public void evaluateAnnDiff(Document doc, String classTypeOriginal,
            String classTypeTest, String classFeat, Set labelSet, HashMap labels2MR) {
        // for each label
        AnnotationSet annsOriginal = getInputAS(doc).get(classTypeOriginal);
        AnnotationSet annsTest = getInputAS(doc).get(classTypeTest);
        HashSet annsKey = new HashSet();
        HashSet annsRes = new HashSet();
        HashSet signSet = new HashSet();
        signSet.add(classFeat);
        for (Object obj : labelSet) {
            String label = obj.toString();
            annsKey.clear();
            annsRes.clear();
            for (Object objAnn : annsOriginal) {
                if (((FeatureMap) ((Annotation) objAnn).getFeatures()).get(classFeat) != null) {
                    if (((FeatureMap) ((Annotation) objAnn).getFeatures()).get(classFeat).toString().equals(label)) {
                        annsKey.add(objAnn);
                    }
                }
            }
            for (Object objAnn : annsTest) {
                if (((FeatureMap) ((Annotation) objAnn).getFeatures()).get(classFeat) != null) {
                    if (((FeatureMap) ((Annotation) objAnn).getFeatures()).get(classFeat).toString().equals(label)) {
                        annsRes.add(objAnn);
                    }
                }
            }
            AnnotationDiffer annDiff = new AnnotationDiffer();
            annDiff.setSignificantFeaturesSet(signSet);
            annDiff.calculateDiff(annsKey, annsRes);
            EvaluationMeasuresComputation emc = (EvaluationMeasuresComputation) labels2MR.get(label);
            emc.correct += annDiff.getCorrectMatches();
            emc.partialCor += annDiff.getPartiallyCorrectMatches();
            emc.missing += annDiff.getMissing();
            emc.spurious += annDiff.getSpurious();
        }
    }

    /**
     * Evaluate the test document by comparing the annotations with the golden
     * standard
     */
    public void evaluateAnnotations(Document doc, String classTypeOriginal,
            String classTypeTest, String classFeat, Set labelSet, HashMap labels2MR) {
        AnnotationSet annsOriginal = getInputAS(doc).get(classTypeOriginal);
        AnnotationSet annsTest = getInputAS(doc).get(classTypeTest);
        HashSet annsKey = new HashSet();
        HashSet annsRes = new HashSet();
        // For each label
        for (Object obj : labelSet) {
            String label = obj.toString();
            annsKey.clear();
            annsRes.clear();
            for (Object objAnn : annsOriginal) {
                Object label0 = ((Annotation) objAnn).getFeatures().get(classFeat);
                if (label0 != null && label0.equals(label)) {
                    annsKey.add(objAnn);
                }
            }
            for (Object objAnn : annsTest) {
                Object label0 = ((Annotation) objAnn).getFeatures().get(classFeat);
                if (label0 != null && label0.equals(label)) {
                    annsRes.add(objAnn);
                }
            }
            EvaluationMeasuresComputation measuresOfResults = (EvaluationMeasuresComputation) labels2MR.get(label);
            for (Object annOrig : annsKey) {
                boolean isMatch = false;
                for (Object annTest : annsRes) {
                    if (((Annotation) annTest).coextensive(((Annotation) annOrig))) {
                        measuresOfResults.correct++;
                        isMatch = true;
                        break;
                    }
                }
                if (!isMatch) {
                    measuresOfResults.missing++;
                }
            }
            for (Object annTest : annsRes) {
                boolean isMatch = false;
                for (Object annOrig : annsKey) {
                    if (((Annotation) annTest).coextensive(((Annotation) annOrig))) {
                        isMatch = true;
                        break;
                    }
                }
                if (!isMatch) {
                    measuresOfResults.spurious++;
                }
            }
        }
    }

    /**
     * Evaluate the test document by comparing the annotations with the golden
     * standard, for Relation learning
     */
    public void evaluateAnnotationsRel(Document doc, String classTypeOriginal,
            String classTypeTest, String classFeat, String arg1F, String arg2F,
            Set labelSet, HashMap labels2MR) {
        AnnotationSet annsOriginal = getInputAS(doc).get(classTypeOriginal);
        AnnotationSet annsTest = getInputAS(doc).get(classTypeTest);
        HashSet annsKey = new HashSet();
        HashSet annsRes = new HashSet();
        // For each label
        for (Object obj : labelSet) {
            String label = obj.toString();
            annsKey.clear();
            annsRes.clear();
            for (Object objAnn : annsOriginal) {
                Object label0 = ((Annotation) objAnn).getFeatures().get(classFeat);
                if (label0 != null && label0.equals(label)) {
                    annsKey.add(objAnn);
                }
            }
            for (Object objAnn : annsTest) {
                Object label0 = ((Annotation) objAnn).getFeatures().get(classFeat);
                if (label0 != null && label0.equals(label)) {
                    annsRes.add(objAnn);
                }
            }
            EvaluationMeasuresComputation measuresOfResults = (EvaluationMeasuresComputation) labels2MR.get(label);
            for (Object annOrig : annsKey) {
                Object arg1VO = ((Annotation) annOrig).getFeatures().get(arg1F);
                Object arg2VO = ((Annotation) annOrig).getFeatures().get(arg2F);
                boolean isMatch = false;
                for (Object annTest : annsRes) {
                    FeatureMap feats = ((Annotation) annTest).getFeatures();
                    if (arg1VO.equals(feats.get(arg1F)) && arg2VO.equals(feats.get(arg2F))) {
                        measuresOfResults.correct++;
                        isMatch = true;
                        break;
                    }
                }
                if (!isMatch) {
                    measuresOfResults.missing++;
                }
            }
            for (Object annTest : annsRes) {
                Object arg1VO = ((Annotation) annTest).getFeatures().get(arg1F);
                Object arg2VO = ((Annotation) annTest).getFeatures().get(arg2F);
                boolean isMatch = false;
                for (Object annOrig : annsKey) {
                    FeatureMap feats = ((Annotation) annOrig).getFeatures();
                    if (arg1VO.equals(feats.get(arg1F)) && arg2VO.equals(feats.get(arg2F))) {
                        isMatch = true;
                        break;
                    }
                }
                if (!isMatch) {
                    measuresOfResults.spurious++;
                }
            }
        }
    }

    private AnnotationSet getInputAS(Document doc) {
        if (inputASName == null || inputASName.trim().length() == 0) {
            return doc.getAnnotations();
        } else {
            return doc.getAnnotations(inputASName);
        }
    }
}
